/**
 * 
 */
package process;

import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Vector;

import util.DBConnect;
import util.SetReader;
import util.Timer;

import data.SimilaritySet;

/**
 * @author yexijiang
 * @date Jul 3, 2008
 */
public class PreProcess 
{
	private static String targetTags[];// = {	"programming", "web2.0", "photography", "shopping", "Linux",
										//			"education", "news", "Blogs", "games",	"development"	};
	private static String targetTagsId[];// = {	"10", "12", "15", "17", "18", "20", "23", "24", "27", "30"	};
	
	private int m_SetDimension;
	private int m_NumofRecordsPerSets;
	private int m_NumofCandidate;
	
	private double m_Maxs[];
	private double m_Mins[];
	private double m_Diff[];
	
	private String m_DmUrl = "jdbc:mysql://localhost:3306/dm";
	private String m_TagUrl = "jdbc:mysql://localhost:3306/tag";
	private String m_Usr = "root";
	private String m_Pwd = "";
	
	private SimilaritySet m_Sets[];
	private int m_SimSequence[];	//	the sequence of the similarity set be used
	
	public PreProcess(int numOfCandidate, int sequence[], int numRecords)
	{
		m_NumofCandidate = numOfCandidate;
		m_SimSequence = sequence;
		m_SetDimension = sequence.length;
		m_NumofRecordsPerSets = numRecords;
		m_Maxs = new double[m_SetDimension];
		m_Mins = new double[m_SetDimension];
		m_Diff = new double[m_SetDimension];

	}
	
	public SimilaritySet[] preProcess()
	{
		long startPreProcess = Timer.startRecord();
		readTagCandidates(m_NumofCandidate);
		System.out.println("Begin read raw data");
		long startReadRawData = Timer.startRecord();
		readRawData(m_SimSequence);
		long elapseReadRawData = Timer.endRecord(startReadRawData);
		System.out.println("End read raw data, cost time:" + elapseReadRawData);
		
		if(m_SetDimension != 1)
		{
			System.out.println("Begin normalization");
			long startNormalization = Timer.startRecord();
			minMaxNormalize();
			long elapseNormalization = Timer.endRecord(startNormalization);
			System.out.println("End normalization, cost time:" + elapseNormalization);
		

			System.out.println("Begin fill in missing value");
			long startHandleMissingValue = Timer.startRecord();
			fillingValue();
			handleMissingValue();
			long elapseHandleMissingValue = Timer.endRecord(startHandleMissingValue);
			System.out.println("End fill in missing value, cost time:" + elapseHandleMissingValue);
		}
		long elapsePreProcess = Timer.endRecord(startPreProcess);
		System.out.println("The whole elapse is:" + elapsePreProcess);
		
		return m_Sets;
	}
	
	/**
	 * Read the candidate tag from the database
	 * @param num	the number of the candidate tags
	 */
	public void readTagCandidates(int num)
	{
		Connection conn = DBConnect.connect(m_TagUrl, m_Usr, m_Pwd);
		Statement stmt = null;
		
		targetTags = new String[num];
		targetTagsId = new String[num];
		m_Sets = new SimilaritySet[targetTags.length];
		
		try {
			stmt = conn.createStatement();
			String queryTag = "select tag, tag_id_l as tag_id, count(distinct tag_id_r) as counts " +
								"from dm.sim_tag1, tag.tag_dict " +
								"where tag.tag_dict.tag_id=dm.sim_tag1.tag_id_l and tag not in " +
								"(select tag from tag.trashtag) " +
								"group by tag_id_l order by counts desc limit " + num + ";";
			ResultSet rs = stmt.executeQuery(queryTag);
			
			int i = 0;
			while(rs.next())
			{
				targetTagsId[i] = rs.getString("tag_id");
				targetTags[i] = rs.getString("tag");
				++i;
			}
			conn.close();
		} catch (SQLException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		
	}
	
	public void insertTargetTags()
	{
		String url = "jdbc:mysql://localhost:3306/tag_clustering";
		Connection conns = DBConnect.connect(url, m_Usr, m_Pwd);
		Statement stmts = null;
		
		try{
			stmts = conns.createStatement();
			for(int i = 0; i < targetTags.length; ++i)
			{
				String insert = "insert into target_tag(tagname) values('" + targetTags[i] + "');";
				stmts.executeUpdate(insert);
			}
		} catch(SQLException sqle)
		{
			sqle.printStackTrace();
		}
	}
	
	/**
	 * Read correlated tags for each targeted tag
	 */
	private void readRawData(int sequence[])
	{
		//int sequence[] = {1};
		SetReader reader = new SetReader(m_NumofRecordsPerSets, sequence);
		for(int i = 0; i < targetTags.length; ++i)
		{
			m_Sets[i] = reader.readSets(targetTagsId[i]);
		}
	}
	
	/**
	 * Normalize the data using min-max normalization
	 */
	private void minMaxNormalize()
	{
		//	read all maxs and mins from database

		Connection conn = DBConnect.connect(m_DmUrl, m_Usr, m_Pwd);
		Statement stmt = null;
		try {
			stmt = conn.createStatement();
			for(int i = 0; i < m_SetDimension; ++i)
			{
				String queryMax = "select max(sim) as max from sim_tag" + (i + 1);
				String queryMin = "select min(sim) as min from sim_tag" + (i + 1);
				ResultSet rsMax = stmt.executeQuery(queryMax);
				while(rsMax.next())
				{
					m_Maxs[i] = rsMax.getDouble("max");
					m_Diff[i] = rsMax.getDouble("max");
				}
				ResultSet rsMin = stmt.executeQuery(queryMin);
				while(rsMin.next())
				{
					m_Mins[i] = rsMin.getDouble("min");
					m_Diff[i] -= rsMin.getDouble("min");
				}
			}
		} catch (SQLException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		
		//	normalize each dimension
		for(int i = 0; i < m_Sets.length; ++i)	//	for each target tag set
		{
			Vector<Double> vectors[] = new Vector[m_Sets[i].size()];
			Vector<String> keys = m_Sets[i].getKeys();
			for(int j = 0; j < vectors.length; ++j)	//	for each vector of the set
			{
				vectors[j] = m_Sets[i].getRecordByKeys(keys.get(j));

				for(int k = 0; k < vectors[j].size(); ++k)
				{
					Double newValue = vectors[j].get(k);
					//	if this place is null, let it be
					if(newValue == null)
						continue;
					newValue = (newValue - m_Mins[k]) / m_Diff[k];
					newValue = (double)Math.round(newValue * 10000000) / 10000000;
					vectors[j].set(k, newValue);
				}
				m_Sets[i].updateVector(keys.get(j), vectors[j]);
			}
		}
		
	}
	

	/**
	 * Filling the null value with 0, and remove the useless vector.
	 */
	private void fillingValue()
	{
		//	filling values
		int missThreash = m_SetDimension / 2;
		
		for(int i = 0; i < m_Sets.length; ++i)	//	for each target tag set
		{
			Vector<Double> vectors[] = new Vector[m_Sets[i].size()];
			Vector<String> keys = m_Sets[i].getKeys();
			Vector<Integer> trashTag = new Vector<Integer>();
			for(int j = 0; j < vectors.length; ++j)	//	for each vector of the set
			{
				vectors[j] = m_Sets[i].getRecordByKeys(keys.get(j));
				
				int missNum = 0;
				boolean badVec = false;
				double avg = 0.0; 
				
				for(int k = 0; k < m_SetDimension; ++k)
				{
					if(vectors[j].get(k) == null)
					{
						++missNum;
						if(missNum > missThreash)
						{
							badVec = true;
						}
					}
				}
				
				if(badVec)	//	if it is a bad vector
				{
					m_Sets[i].removeRecord(keys.get(j));
					trashTag.add(j);
					continue;
				}
				

				for(int k = 0; k < vectors[j].size(); ++k)
				{
					Double newValue = vectors[j].get(k);
					//	if this place is null, let it be
					if(newValue == null)
					{
						newValue = 0.0;
						vectors[j].set(k, newValue);
					}
				}
				m_Sets[i].updateVector(keys.get(j), vectors[j]);
			}
			for(int j = 0; j < trashTag.size(); ++j)
			{
				for(int k = 0; k < trashTag.size(); ++k)
				{
					if(trashTag.get(k) > trashTag.get(j))
					{
						int iV = trashTag.get(k);
						trashTag.set(k, --iV);
					}
				}
				m_Sets[i].removeKeyByIndex(trashTag.get(j));
			}
		}
	}
	
	/**
	 * Fill in the missing value, value = average value of the tag in other sets,
	 * called after normalization
	 */
	private void handleMissingValue()
	{
		
		//	handling missing values
		int missThreash = m_SetDimension / 2;
		
		for(int i = 0; i < m_Sets.length; ++i)	//	for each target tag set
		{
			double means[] = new double[m_SetDimension];
			double weightMatrix[][] = new double[m_SetDimension][m_SetDimension];
			
			Vector<Double> vectors[] = new Vector[m_Sets[i].size()];
			Vector<String> keys = m_Sets[i].getKeys();
			
			for(int j = 0; j < vectors.length; ++j)
			{
				vectors[j] = m_Sets[i].getRecordByKeys(keys.get(j));
			}
			
			//	calculate the means
			for(int j = 0; j < means.length; ++j)
			{
				for(int k = 0; k < vectors.length; ++k)
				{
					means[j] += vectors[k].get(j);
				}
				means[j] /= vectors.length;
				//System.out.println("For target tag '" + m_Sets[i].getTargetTag() + "', the mean for dimension " + j + " is " + means[j]);
			}
			
			for(int j = 0; j < m_SetDimension; ++j)
			{
				for(int k = 0; k < m_SetDimension; ++k)	//covMatrix(j,k)
				{
					double dominatorX = 0.0, dominatorY = 0.0;
					
					for(int l = 0; l < vectors.length; ++l)
					{
						weightMatrix[j][k] += (vectors[l].get(j) - means[j]) * (vectors[l].get(k) - means[k]);
						dominatorX += Math.pow(vectors[l].get(j) - means[j], 2);
						dominatorY += Math.pow(vectors[l].get(k) - means[k], 2);
					}
					dominatorX = Math.sqrt(dominatorX);
					dominatorY = Math.sqrt(dominatorY);
					weightMatrix[j][k] /= (dominatorX * dominatorY);
					
					weightMatrix[j][k] = (double)Math.round(weightMatrix[j][k] * 10000000) / 10000000;
				}
			}
			
			//	normalize the pearsonMatrix
			for(int j = 0; j < weightMatrix.length; ++j)
			{
				double max = weightMatrix[j][0], min = weightMatrix[j][0];
				for(int k = 0; k < weightMatrix[j].length; ++k)
				{
					if(weightMatrix[j][k] < min)
						min = weightMatrix[j][k];
				}
				
				double sum = 0.000001;	//	make sure that no value is 0
				for(int k = 0; k < weightMatrix[j].length; ++k)
				{
					weightMatrix[j][k] += Math.abs(min);
					sum += weightMatrix[j][k];

				}
				
				for(int k = 0; k < weightMatrix[j].length; ++k)
				{
					weightMatrix[j][k] /= sum;
					//System.out.print("" + weightMatrix[j][k] + "\t");
				}
				//System.out.println();
			}
			//System.out.println();
			
			for(int j = 0; j < vectors.length; ++j)	//	for each vector of the set
			{
				
				int missNum = 0;
				boolean badVec = false;
				
				/*double avg = 0.0; 
				
				
				for(int k = 0; k < m_SetDimension; ++k)
				{
					if(vectors[j].get(k) == 0)
					{
						++missNum;
						if(missNum > missThreash)
						{
							badVec = true;
						}
					}
					else	//	calculate the average value
					{
						for(int l = 0; l < m_SetDimension; ++l)
						{
							avg += weightMatrix[k][l] * vectors[j].get(l);
						}
					}
				}*/
				

				//avg /= (m_SetDimension - missNum);

				for(int k = 0; k < vectors[j].size(); ++k)
				{
					Double newValue = vectors[j].get(k);
					//	if this place is null, let it be
					if(newValue == 0)
					{
						double avg = 0.0;
						for(int l = 0; l < m_SetDimension; ++l)
						{
							avg += weightMatrix[k][l] * vectors[j].get(l);
						}
						newValue = avg;
						newValue = (double)Math.round(newValue * 10000000) / 10000000;
						vectors[j].set(k, newValue);
					}
				}
				m_Sets[i].updateVector(keys.get(j), vectors[j]);
			}
		}
	}
	
	public String toString()
	{
		String msg = new String();
		for(int i = 0; i < m_Sets.length; ++i)
		{
			msg += "For tag '" + targetTags[i] + "'\n";
			msg += m_Sets[i];
			msg += "\n\n\n";
		}
		return msg;
	}
	
	public static void main(String[] args)
	{
		//int sequence[] = {1, 2, 3, 4, 5, 6, 7, 8};
		int sequence[] = {4,5,6};
		int numOfTargetTags = 100;
		int numOfRecordsPerTargetTag = 30;
		PreProcess pre = new PreProcess(numOfTargetTags, sequence , numOfRecordsPerTargetTag);
		pre.readTagCandidates(numOfTargetTags);
		pre.insertTargetTags();
		//pre.preProcess();
		//System.out.println(pre);
	}
}
